/***************************************************************************\
 *  webgrab - v1.5 - Copyright 1995-2006, Brian J. Swetland et al          *
 *                                                                         *
 *  - initial version by Brian Swetland                                    *
 *  - cleaned up a bit by Brandon Long                                     *
 *  - proxy support by Kristin Buxton                                      *
 *  - cleaned up more by Brian Swetland                                    *
 *  - HTTP/1.1 support (Host: header) added by Ari Gordon-Schlosberg       *
 *  - User authentication by Dave Terrell (dbt@meat.net)                   * 
 *  - Fixed command line parsing by David Terrell                          *
 *  - Added POST support (notarus@uiuc.edu)                                *
 *  - Added Referer: url cl option (no self-headers needed) (notarus)      *
 *                                                                         *
 *  This program has no warranty, expressed or implied.                    *
 *                                                                         *
 *  Redistribution in source or binary form, modified or unmodified, is    *
 *  permitted as long as the above notices are retained.                   *
\***************************************************************************/

#define VERSION "1.5"
#define PROTOCOL_VERSION "1.0"
#include <stdio.h>
#include <fcntl.h>

#ifdef __bsdi__
# include <sys/malloc.h>
#else
# ifndef NeXT
#  include <malloc.h>
# endif
#endif

#include <sys/time.h>
#include <sys/types.h>

#include <sys/socket.h>
#include <netinet/in.h>
#include <unistd.h>
#include <netdb.h>

#include <string.h>


/* strdup isn't portable, so we make our own.  */
char *strd(char *s) {
    char *d;
 
    d = (char *) malloc(strlen(s) + 1);
    strcpy(d,s);
    return(d);
}


/* Will get character #pos from string str of length len, unless pos is
   past the end in which case return 0 */
#define retch(str,len,pos) (pos>len ? 0 : str[pos])

/* Takes a string and base64 encodes it, returning a malloced string.
   Caller takes responsibility to free it. */
char *base64(char *inc)
{
    char *out;
    int outl,incl,i,b;
    char c;
    
    static const char lookup[]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

    incl = strlen(inc);
    outl = ((incl - 1) / 3 + 1) * 4;
    out = malloc(outl+1);
    for(i=0;i<outl;i++)
    {
        b = i / 4 * 3;
        switch(i % 4)
        {
        case 0:
            out[i] = lookup[(retch(inc,incl,b) & 0xFC) >> 2];
            break;
        case 1:
            out[i] = lookup[((retch(inc,incl,b) & 0x03) << 4) |
                           ((retch(inc,incl,b+1) & 0xF0) >> 4)];
            break;
        case 2:
            if(c = retch(inc,incl,b+1)){
                out[i] = lookup[((c & 0x0F) << 2) |
                               ((retch(inc,incl,b+2) & 0xC0) >> 6)];
            }
            else
                out[i] = '='; /* string is over, encoding is done, thought
                                 I'd something more to say. */
            break;
        case 3:
            if(c = retch(inc, incl, b+2)){
                out[i] = lookup[(c & 0x3F)];
            }
            else
                out[i] = '='; /* yeah we're done */
            break;
        }   
    }
    return(out);
}

/* parses URL looking like blah://host[:port][/path]
   will ignore anything before the first : and terminate path when it
   hits >, ", or whitespace -- returns portno or 0 if bad url */

int parseURL(char *url, char **host, char **path)
{
    char *p, *pp;
    int port;
    
    p = url;  
    
        /* skip anything up to the first : (the one after http, etc) */
    while(*p && *p!=':') p++;
    if(!*p) 
        return 0;
    
        /* REQUIRE two '/'s */
    if(!(*(++p) && (*p =='/') && *(++p) && (*p == '/')))
        return 0;
    
    p++;
    
        /* mark the beginning of the hostname */
    pp = p;
        /* hostname is terminated by a '/' or '>','"',or whitespace */
    while(*p && *p!=':' && *p!='/' && *p!='"' && *p!='>' && !isspace(*p)) 
        p++;
    
    *host = (char *) malloc(p-pp+1);
    strncpy(*host,pp,p-pp);
    (*host)[p-pp]=0;
    
        /* optionally read a portnumber */
    if(*p==':'){
        p++;
        port = 0;
        while(*p && isdigit(*p)){
            port = port*10 + (*p-'0');
            p++;
        }
        if(!*p || *p!='/') {
            free(*host);
            return 0;
        }
    } else {
        port = 80;
    }
    
        /* still more */
    if(*p && (*p=='/')){
        pp = p;
        while(*p && *p!='"' && *p!='>' && !isspace(*p)) p++;
        *p = 0;
        *path = strd(pp);
    } else {
        *path = strd("/");
    }
    return port;
}

void usage(char *argv) 
{
    printf("\nWebgrab: The Command Line Browser\tVersion %s \n",VERSION);
    printf("Usage: %s [-shrt] [-p <proxy>] [-a <user>] -e <url> <url>\n",argv);
    printf("   -s      Suppress Headers\n");
    printf("   -h      Headers Only\n");
    printf("   -r      Read HTTP headers from stdin\n");
    printf("   -t      Use HTTP POST request (incompatible with -h or -r)\n");
    printf("           POSTed data is read from stdin.\n");
    printf("   -p      Next argument is <proxy>\n");
    printf("   -a      Next argument is <user> or <user:pass> for auth\n");
    printf("   -e      Next argument <url> should be passed as the referer url\n");
    printf("   <proxy> HTTP Proxy Host ( hostname[:port] format )\n");
    printf("   <url>   URL to retrieve (in http:// format)\n\n");
    exit(1);
}

int main(int argc, char *argv[])
{
    int s, i, port, pport,contentlength;
    struct sockaddr_in sa;
    struct hostent *hp;
    FILE *fpo,*fpi;
    char buf[1024];
    char *path,*host,*p,*u,*contentstore,*referer;
    extern char *optarg;
    extern int optind, opterr, optopt;
    char c;
    
      /* operational flags */
    int ignore=0,head=0,post=0,readin=0,proxy=0,url=0,user=0,chunked=0;
    char *proxyhost, *authstr;
    char *request; /* what type of request?  GET, POST, or HEAD */

    /*setup */
    contentstore=NULL; referer=NULL;

    /* options processing */
    while((c = getopt(argc, argv, "shrte:p:a:")) != EOF) switch (c)
    {
    case 's':
        ignore = 1;
        break;
    case 'h':
        if(post)
            usage(argv[0]);
        head = 1;
        break;
    case 't':
        if(head || readin)
            usage(argv[0]);
        post = 1;
	break;
    case 'r':
      if (post)
	 usage(argv[0]);
        readin = 1;
        break;
    case 'p':
        proxy = 1;
        proxyhost = optarg;
        break;
    case 'a':
        user = 1;
        authstr = optarg;
        break;
    case 'e':
        referer = optarg;
        break;
    case '?':
    default:
        usage(argv[0]);
        break;
    }
    
    if(optind<argc)
        url=optind;
        
    if(!url) usage(argv[0]);
    
    if(!(port=parseURL(argv[url], &host, &path))){
        fprintf(stderr,"error: invalid url\n");
        exit(1);
    }

    
  /* find the server */
    if(proxy){
        pport = 8080;
        p = proxyhost;

            /* look for a portnum */
        while(*p){
            if(*p==':'){
                *p=0;
                p++;
                pport = atoi(p);
                break;
            }
            p++;
        }
        if(!(hp = gethostbyname(proxyhost))) {
            fprintf(stderr,"error: can't get proxy %s.\n",proxyhost);
            exit(1);
        }
    } else {
        if(!(hp = gethostbyname(host))) {
            fprintf(stderr,"error: can't get host %s.\n",host);
            exit(1);
        }
    }

        /* Check user / user:pass syntax for Authorization: header. */
    if(user){
        p = authstr;
        u = NULL;
        while(*p){
            if(*p==':' && *(p+1)){
                    /* we're done, we've got the u:p 
                     * from cmd line.  Don't change authstr */
                u = authstr;
                break;
            }
            p++;
        }
        if(u==NULL)
        {
                /* no password on cmd line?  get one. */
            u = getpass("Enter password: ");
            if(u==NULL){
                fprintf(stderr, "Can't get password\n");
                exit(1);
            }
        p = malloc(strlen(p) + strlen(u)+2);
        sprintf(p, "%s:%s", authstr, u);
        u = p;
        }
    }

    
        /* Setup the socket */
    memset(&sa, 0, sizeof(sa));  
    sa.sin_port = htons(proxy ? pport : port);
    memcpy((char *)&sa.sin_addr, (char *)hp->h_addr, hp->h_length);
    sa.sin_family = hp->h_addrtype;
  
        /* allocate the socket */
    if((s = socket(hp->h_addrtype, SOCK_STREAM, 0)) < 0){
        fprintf(stderr,"error: can't get socket\n");
        exit(1);
    }
    
        /* connect to the server */
    if(connect(s, &sa, sizeof(sa)) < 0){
        close(s);
        fprintf(stderr,"error: can't connect\n");
        exit(1);
    }
   
     fpo = fdopen(s,"w");
    fpi = fdopen(s,"r");
    if (head) {
        request = "HEAD";
    } else {
        if (post) {
            request = "POST";
        } else {
            request = "GET";
        }
    }
    if(proxy){
        fprintf(fpo,"%s http://%s:%d%s HTTP/%s\r\n",request,host,port,path,PROTOCOL_VERSION);
    } else {
        fprintf(fpo,"%s %s HTTP/%s\r\n",request,path,PROTOCOL_VERSION);
    }
    fprintf(fpo,"Host: %s",host);
    if (port != 80){ 
      /* fix for name-based virt servers not on default port -man */
      fprintf(fpo,":%d",port);
    }
    fprintf(fpo,"\n");
    fprintf(fpo,"Connection: close\n"); /* Since many servers assume you
                                         * want to keep the connection open
                                         *  -dbt 9/29/98 */
        /* basic auth */
    if(user){
        p = base64(u);
        fprintf(fpo, "Authorization: Basic %s\n", p);
        free(p);
    }   
    
    if (readin) {
            /* copy headers from stdin ... */
        while(!feof(stdin)){
            i = fread(buf,1,1024,stdin);
            if(i) fwrite(buf,1,i,fpo);
            if(feof(stdin)) break;
        } 
    } else {
            /* send our normal header info */
        fprintf(fpo,
                "User-Agent: WebGrab/%s (commandline forever)\r\n",
                VERSION);
	if (referer)
	  fprintf(fpo,"Referer: %s\r\n",referer);

    }      

    if (post){
    /* first, figure out how long the content is. Fortunately, 
       this has to be one line of data. */
      contentlength=0;
      contentstore=(char *)malloc(1);
      contentstore[0]=0;
      while(!feof(stdin)){
            i = fread(buf,1,1024,stdin);
	    contentlength+=i;
/* 	    printf("Read in %d chars, contentlength=%d, got %s\n",i,contentlength,buf); */
	    realloc(contentstore,contentlength+1);
	    strncat(contentstore,buf,i);
            if(feof(stdin)) break;
      }
      fprintf(fpo,"Content-type: application/x-www-form-urlencoded\r\n");
      fprintf(fpo,"Content-length: %d\r\n\r\n",contentlength); 
      fwrite(buf,1,contentlength,fpo);
      fputs("\r\n",fpo);
    }
      fputs("\r\n",fpo);
      fflush(fpo);



    
        /* handle headers */
    while(!feof(fpi)){
        fgets(buf,1024,fpi);
        if(!ignore) fprintf(stdout,"%s",buf);
        if(feof(fpi) || buf[0]<' ') break;
    }
    while(!feof(fpi)){
        i = fread(buf,1,1024,fpi);
        if(i) fwrite(buf,1,i,stdout);
        if(feof(fpi)) break;
    }
    close(s);
    exit(0);
}
